package com.cyc.basic;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

/**
 * @description: 读取日志并去重
 * @version 1.0
 * @author cyc
 * @date 2021/12/1 15:01
 */
public class ReadLogTest {
    public static void main(String[] args) {
        // resources文件夹下
        InputStream inputStream = ReadLogTest.class.getClassLoader().getResourceAsStream("data/idp.log");
        List<String> userIdList = new ArrayList<>();
        InputStreamReader reader;
        BufferedReader bufferedReader;
        try {
            assert inputStream != null;
            reader = new InputStreamReader(inputStream);
            bufferedReader = new BufferedReader(reader);
            String line;
            while ((line = bufferedReader.readLine()) != null) {
                // 起始标志
                String startSign = "usereId = ";
                int start = line.indexOf(startSign);
                // 结束标志
                String endSign = "";
                int endIndex = line.indexOf(endSign);
                // 不存在该关键词,跳过,截止下一行
                if (start <= 0 || endIndex <= 0) {
                    continue;
                }
                int startIndex = start + startSign.length();
                // 从开始位置截取到结束位置
                String userId = line.substring(startIndex, endIndex);
                userIdList.add(userId);
            }
            System.out.println("去重前的总数为: " + userIdList.size());
            System.out.println("去重前的数据为: "+userIdList);
            long distinctCount = userIdList.stream().distinct().count();
            System.out.println("去重后的总数为: " + distinctCount);
            System.out.println("去重后的数据为" + userIdList.stream().distinct().collect(Collectors.toList()));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
